library(ggplot2)
library(ggpubr)
library(CDM)
library(boot)
library(tidyverse)
library(dummy)
dummy 0.1.3
dummyNews()
library(stringi)
library(stringr)
rm(list = ls())

x_pre <- read_csv("../data/FirstYearProject/OUTPUT.csv")
Parsed with column specification:
cols(
  .default = col_character(),
  SubjectID = col_double(),
  `Auto Score 1` = col_double(),
  `Auto Score 2` = col_double(),
  `Auto Score 3` = col_double(),
  `Auto Score 4` = col_double(),
  `Auto Score 5` = col_double(),
  `Auto Score 6` = col_double(),
  `Auto Score 7` = col_double(),
  `Auto Score 8` = col_double(),
  `Auto Score 9` = col_double(),
  `Auto Score 10` = col_double(),
  `Auto Score 11` = col_double(),
  `Auto Score 12` = col_double(),
  `Auto Score 13` = col_double(),
  `Auto Score 14` = col_double(),
  `Auto Score 15` = col_double(),
  `Auto Score 16` = col_double(),
  `Auto Score 17` = col_double(),
  `Auto Score 18` = col_double(),
  `Auto Score 19` = col_double()
  # ... with 34 more columns
)
See spec(...) for full column specifications.

Q_from_book <- read_csv("../data/FirstYearProject/final_result_similar.csv") %>% mutate(`Learning Objective` = `Topic`)
Parsed with column specification:
cols(
  Question = col_character(),
  Option1 = col_character(),
  Option2 = col_character(),
  Option3 = col_character(),
  Option4 = col_character(),
  Answer = col_character(),
  `Learning Objective` = col_character(),
  Topic = col_character(),
  `Difficulty Level` = col_character(),
  `Skill Level` = col_character(),
  `APA Learning Objective` = col_character()
)
Q_from_book <- Q_from_book %>% 
  mutate(`Learning Objective` = str_trim(str_remove_all(`APA Learning Objective`, "\\."))) %>% 
  filter(`Learning Objective` != "nan")
  

glimpse(Q_from_book)
Observations: 992
Variables: 11
$ Question                 <chr> "Which of the following is an example of social ...
$ Option1                  <chr> "a. You feel guilty because you lied to your tru...
$ Option2                  <chr> "b. When you get hungry, you have trouble concen...
$ Option3                  <chr> "c. You didn\u0092t do well on the test because ...
$ Option4                  <chr> "d. You almost fall asleep at the wheel, so you ...
$ Answer                   <chr> "A", "A", "D", "C", "A", "C", "C", "B", "D", "D"...
$ `Learning Objective`     <chr> "11 Describe key concepts, principles, and overa...
$ Topic                    <chr> "Defining Social Psychology", "Defining Social P...
$ `Difficulty Level`       <chr> "Moderate", "Moderate", "Moderate", "Moderate", ...
$ `Skill Level`            <chr> "Understand the Concepts", "Understand the Conce...
$ `APA Learning Objective` <chr> "1.1 Describe key concepts, principles, and over...

Q_from_book %>% distinct(`Skill Level`)
NA
learning_obj <- Q_from_book %>%
  distinct(`Learning Objective`) %>%
  mutate(lo_id = row_number())


Q_pre <- Q_from_book %>% inner_join(learning_obj) %>% select(Question, `Learning Objective`, lo_id) %>% mutate(temp = str_trim(str_replace_all(Question, "_|\\.", "")))
Joining, by = "Learning Objective"
learning_obj


Q_pre <- Q_from_book %>% inner_join(learning_obj) %>% select(Question, `Learning Objective`, lo_id) %>% 
  mutate(temp = str_trim(str_replace_all(Question, "_|\\.", ""))) %>%
  mutate(Q_UNIQUE_ID = row_number()) 
Joining, by = "Learning Objective"
Q_pre
NA

head(x_pre)
NA

x.gather <-x_pre %>% gather(key = "key", value = "value", -File, -SubjectID)
x.gather 
x.questions <- 
  
  x.gather %>% filter(str_detect(key, "Question")) %>%

  anti_join(
    x.gather %>% filter(str_detect(key, "Question")) %>% 
      group_by(File, SubjectID, value) %>% 
      summarise(cnt = n(), question_number = paste(key, collapse = ",")) %>% 
      filter(cnt > 1) %>% ungroup(),
            by = "value"
    ) # Taking out generic questions (having same question text but different answers)


x.questions.dist <- x.questions %>% distinct(value) %>% drop_na() %>%  
  #mutate(Q_UNIQUE_ID = row_number()) %>% 
  mutate(temp = str_trim(str_replace_all(value, "_|\\.", ""))) %>% 
  
  inner_join(
    Q_pre, by = "temp"
    
    
  )



x.questions.dist %>% write_csv("../data/FirstYearProject/Q_distinct_id.csv")
x.questions.dist
Q <- x.questions.dist %>% distinct(Q_UNIQUE_ID, lo_id) %>% arrange(Q_UNIQUE_ID) %>%
  mutate(present = 1) %>%
  
  spread(key = "lo_id", value = "present")

Q %>% 
  mutate_all(function(x) ifelse(is.na(x), 0, x)) %>%  
  write_csv("../data/FirstYearProject/Q.csv")
Q
NA

x.answers <- 
  
  x.gather %>% filter(!str_detect(key, "Question"))

x.answers

#Total Questions presented to students 53 Questions are randomly presented to students

x.questions %>% distinct(key)

x.questions.id <- x.questions %>% inner_join(x.questions.dist) #%>% mutate(Q_UNIQUE_ID  = factor(Q_UNIQUE_ID)) 
Joining, by = "value"
x.questions.id

Filter out Generic Questions

Questions with same text but different Answers


x.questions.id.filterd <- x.questions.id %>% 
  anti_join(
    x.questions.id %>% 
      group_by(File, SubjectID, Question) %>% 
      summarise(cnt = n(), question_number = paste(key, collapse = ",")) %>% 
      filter(cnt > 1) %>% ungroup(),
            by = "Question"
    ) %>% select(-lo_id, -`Learning Objective`)


x.questions.id.filterd
NA

We have the correct Questions. Now we need to add marks of answers against the questions.


X.pre <- x.questions.id %>% mutate(id = str_split(key, " ", simplify = TRUE)[,2]) %>% 
  
  inner_join(
    
    x.answers %>% mutate(id = str_split(key, " ", simplify = TRUE)[,3]), by = c("File", "SubjectID", "id")
    
    ) %>% 
  mutate(value.y = as.integer(value.y)) #%>% 
  #mutate(Q_UNIQUE_ID = as.integer(Q_UNIQUE_ID))

#write_csv(X.pre, "X_Pre.csv")
X.pre
unique(X.pre$Q_UNIQUE_ID)
  [1]  19 182 246 299 118  91 114  88  82   9 259 105  78  13 260  49  17 148 273 288
 [21]  40 166  66 202 225 177 126 204 140 293 116 136 186 223 149 229 122 281 137  64
 [41] 291  67  33  30 271 504 966 480 972 477 981 493 496 460 522 465 403 376 441 354
 [61] 529 514 474 430 439 446 348 416 323 356 473 501 986 515 371 331 497 380 505 368
 [81] 343 520 345 467 369 980 483 412 418 604 955 600 590 562 612 549 605 591 610 557
 [ reached getOption("max.print") -- omitted 839 entries ]


X<- X.pre %>% select(-key.x, -key.y, -value.x, -id, -temp, -lo_id, -`Learning Objective`, -Question ) %>% 
  spread(key = "Q_UNIQUE_ID", value = "value.y")  
  

write_csv(X, "../data/FirstYearProject/X.csv")
X

Let’s run some test to verify X


X %>% select(-File, -SubjectID) %>% summarise_all(sum, na.rm = TRUE)
NA

X %>% gather(key = "QuestionID", value = "Score", -File, -SubjectID)
NA

Filter questions asked in Exam I


library(janitor)
X %>% filter(File == "Exam1Trial1") %>% remove_empty(.,which = "cols")
NA

Questions with good attempt count



question_attempted <- X %>% remove_empty(.,which = "cols") %>% 
  gather(key = "QuestionID", value = "Scores", -File, -SubjectID) %>% 
  group_by(File, QuestionID) %>%
  summarise(total_na = sum(is.na(Scores)), total = n(), total_attempted = total - total_na)

question_attempted <- question_attempted %>% filter(total_attempted >= 8)

question_attempted

#%>% filter(QuestionID == "103")

Filtering out questions with lesser attempts


X_filtered <- X %>% remove_empty(.,which = "cols") %>% 
  gather(key = "QuestionID", value = "Scores", -File, -SubjectID) %>% semi_join(question_attempted, by = c("File", "QuestionID")) %>% 
  spread(key = "QuestionID", value = "Scores")

X_filtered

Take away questions answered less that 5 times per exam

X %>% remove_empty(.,which = "cols") %>% write_csv("../data/FirstYearProject/X.csv")

X_filtered %>% remove_empty(.,which = "cols") %>% write_csv("../data/FirstYearProject/X_filtered.csv")

Write CSVs seperate for each trial to avoid having columns for those questions that were not asked in a trial. This will help to show the true picture of sparsity.


fn.clean <- function (df) {
  return(df %>% remove_empty(.,which = "cols"))
  
}


X.individual.list <- X %>% 
nest(-File, .key = "X_full") %>% 
  mutate(X = map(X_full, fn.clean), 
         Q_full = map(X_full, function(df) return (Q)))

X.individual.list
# A tibble: 8 x 4
  File        X_full              X                   Q_full            
  <chr>       <list>              <list>              <list>            
1 Exam1Trial1 <tibble [74 x 940]> <tibble [74 x 286]> <tibble [939 x 5]>
2 Exam1Trial2 <tibble [57 x 940]> <tibble [57 x 277]> <tibble [939 x 5]>
3 Exam2Trial1 <tibble [66 x 940]> <tibble [66 x 236]> <tibble [939 x 5]>
4 Exam2Trial2 <tibble [67 x 940]> <tibble [67 x 237]> <tibble [939 x 5]>
5 Exam3Trial1 <tibble [47 x 940]> <tibble [47 x 178]> <tibble [939 x 5]>
6 Exam3Trial2 <tibble [78 x 940]> <tibble [78 x 179]> <tibble [939 x 5]>
7 Exam4Trial1 <tibble [64 x 940]> <tibble [64 x 239]> <tibble [939 x 5]>
8 Exam4Trial2 <tibble [72 x 940]> <tibble [72 x 239]> <tibble [939 x 5]>

X %>% filter(File == "Exam1Trial1") %>% remove_empty(.,which = "cols")
NA

Merge with Q


Q
NA

fn.skills <- function (df) {
  
  df <- df %>% remove_empty(.,which = "cols") %>%
  gather(key = "Q_UNIQUE_ID", value = "Score", -SubjectID) %>%
  mutate(Q_UNIQUE_ID = as.integer(Q_UNIQUE_ID)) %>% distinct(Q_UNIQUE_ID) %>%
  
  inner_join(
    Q
    
  ) %>% remove_empty(.,which = "cols")  %>% mutate_all(function(x) ifelse(is.na(x), 0, x))
  
  return(df)
  
}


X.Q <- X.individual.list %>% 
  mutate(Q = map(X, fn.skills))
Joining, by = "Q_UNIQUE_ID"
Joining, by = "Q_UNIQUE_ID"
Joining, by = "Q_UNIQUE_ID"
Joining, by = "Q_UNIQUE_ID"
Joining, by = "Q_UNIQUE_ID"
Joining, by = "Q_UNIQUE_ID"
Joining, by = "Q_UNIQUE_ID"
Joining, by = "Q_UNIQUE_ID"
X.Q 
# A tibble: 8 x 5
  File        X_full             X                  Q_full            Q               
  <chr>       <list>             <list>             <list>            <list>          
1 Exam1Trial1 <tibble [74 x 940~ <tibble [74 x 286~ <tibble [939 x 5~ <tibble [285 x ~
2 Exam1Trial2 <tibble [57 x 940~ <tibble [57 x 277~ <tibble [939 x 5~ <tibble [276 x ~
3 Exam2Trial1 <tibble [66 x 940~ <tibble [66 x 236~ <tibble [939 x 5~ <tibble [235 x ~
4 Exam2Trial2 <tibble [67 x 940~ <tibble [67 x 237~ <tibble [939 x 5~ <tibble [236 x ~
5 Exam3Trial1 <tibble [47 x 940~ <tibble [47 x 178~ <tibble [939 x 5~ <tibble [177 x ~
6 Exam3Trial2 <tibble [78 x 940~ <tibble [78 x 179~ <tibble [939 x 5~ <tibble [178 x ~
7 Exam4Trial1 <tibble [64 x 940~ <tibble [64 x 239~ <tibble [939 x 5~ <tibble [238 x ~
8 Exam4Trial2 <tibble [72 x 940~ <tibble [72 x 239~ <tibble [939 x 5~ <tibble [238 x ~
X %>% filter(File == "Exam2Trial2") %>% remove_empty(.,which = "cols") %>%
  gather(key = "Q_UNIQUE_ID", value = "Score", -File, -SubjectID) %>%
  mutate(Q_UNIQUE_ID = as.integer(Q_UNIQUE_ID)) %>% distinct(Q_UNIQUE_ID) %>%

  inner_join(
    Q, by = "Q_UNIQUE_ID"
    
  ) %>% remove_empty(.,which = "cols") %>% mutate_all(function(x) ifelse(is.na(x), 0, x)) %>% summarise_all(sum)
# A tibble: 1 x 2
  Q_UNIQUE_ID   `1`
        <int> <dbl>
1      113706   236
X %>% filter(File == "Exam1Trial1")
NA

fn.write <- function(File, X_full, X, Q_full, Q) {

  
  print(X)
  X %>% write_csv(paste0("../data/FirstYearProject/",File,"_X.csv"))
  Q %>% write_csv(paste0("../data/FirstYearProject/",File,"_Q.csv"))
  
}

#walk2(X.Q$File, X.Q$data_clean, X.Q$data_Q_skills, fn.write)

pwalk(X.Q, fn.write)
NA
NA
LS0tDQp0aXRsZTogIlIgTm90ZWJvb2siDQpvdXRwdXQ6IGh0bWxfbm90ZWJvb2sNCi0tLQ0KDQoNCg0KYGBge3J9DQoNCmxpYnJhcnkoZ2dwbG90MikNCmxpYnJhcnkoZ2dwdWJyKQ0KbGlicmFyeShDRE0pDQpsaWJyYXJ5KGJvb3QpDQpsaWJyYXJ5KHRpZHl2ZXJzZSkNCmxpYnJhcnkoZHVtbXkpDQpsaWJyYXJ5KHN0cmluZ2kpDQpsaWJyYXJ5KHN0cmluZ3IpDQoNCg0KYGBgDQoNCg0KYGBge3J9DQpybShsaXN0ID0gbHMoKSkNCg0KeF9wcmUgPC0gcmVhZF9jc3YoIi4uL2RhdGEvRmlyc3RZZWFyUHJvamVjdC9PVVRQVVQuY3N2IikNCg0KDQpgYGANCg0KYGBge3J9DQoNClFfZnJvbV9ib29rIDwtIHJlYWRfY3N2KCIuLi9kYXRhL0ZpcnN0WWVhclByb2plY3QvZmluYWxfcmVzdWx0X3NpbWlsYXIuY3N2IikgJT4lIG11dGF0ZShgTGVhcm5pbmcgT2JqZWN0aXZlYCA9IGBUb3BpY2ApDQoNCg0KUV9mcm9tX2Jvb2sgPC0gUV9mcm9tX2Jvb2sgJT4lIA0KICBtdXRhdGUoYExlYXJuaW5nIE9iamVjdGl2ZWAgPSBzdHJfdHJpbShzdHJfcmVtb3ZlX2FsbChgQVBBIExlYXJuaW5nIE9iamVjdGl2ZWAsICJcXC4iKSkpICU+JSANCiAgZmlsdGVyKGBMZWFybmluZyBPYmplY3RpdmVgICE9ICJuYW4iKQ0KICANCg0KZ2xpbXBzZShRX2Zyb21fYm9vaykNCg0KYGBgDQoNCmBgYHtyfQ0KDQpRX2Zyb21fYm9vayAlPiUgZGlzdGluY3QoYFNraWxsIExldmVsYCkNCg0KYGBgDQoNCg0KYGBge3J9DQpsZWFybmluZ19vYmogPC0gUV9mcm9tX2Jvb2sgJT4lDQogIGRpc3RpbmN0KGBMZWFybmluZyBPYmplY3RpdmVgKSAlPiUNCiAgbXV0YXRlKGxvX2lkID0gcm93X251bWJlcigpKQ0KDQoNClFfcHJlIDwtIFFfZnJvbV9ib29rICU+JSBpbm5lcl9qb2luKGxlYXJuaW5nX29iaikgJT4lIHNlbGVjdChRdWVzdGlvbiwgYExlYXJuaW5nIE9iamVjdGl2ZWAsIGxvX2lkKSAlPiUgbXV0YXRlKHRlbXAgPSBzdHJfdHJpbShzdHJfcmVwbGFjZV9hbGwoUXVlc3Rpb24sICJffFxcLiIsICIiKSkpDQoNCg0KbGVhcm5pbmdfb2JqDQpgYGANCg0KYGBge3J9DQoNCg0KUV9wcmUgPC0gUV9mcm9tX2Jvb2sgJT4lIGlubmVyX2pvaW4obGVhcm5pbmdfb2JqKSAlPiUgc2VsZWN0KFF1ZXN0aW9uLCBgTGVhcm5pbmcgT2JqZWN0aXZlYCwgbG9faWQpICU+JSANCiAgbXV0YXRlKHRlbXAgPSBzdHJfdHJpbShzdHJfcmVwbGFjZV9hbGwoUXVlc3Rpb24sICJffFxcLiIsICIiKSkpICU+JQ0KICBtdXRhdGUoUV9VTklRVUVfSUQgPSByb3dfbnVtYmVyKCkpIA0KUV9wcmUNCg0KYGBgDQoNCg0KDQpgYGB7cn0NCg0KaGVhZCh4X3ByZSkNCg0KYGBgDQoNCmBgYHtyfQ0KDQp4LmdhdGhlciA8LXhfcHJlICU+JSBnYXRoZXIoa2V5ID0gImtleSIsIHZhbHVlID0gInZhbHVlIiwgLUZpbGUsIC1TdWJqZWN0SUQpDQp4LmdhdGhlciANCmBgYA0KDQoNCg0KYGBge3J9DQp4LnF1ZXN0aW9ucyA8LSANCiAgDQogIHguZ2F0aGVyICU+JSBmaWx0ZXIoc3RyX2RldGVjdChrZXksICJRdWVzdGlvbiIpKSAlPiUNCg0KICBhbnRpX2pvaW4oDQogICAgeC5nYXRoZXIgJT4lIGZpbHRlcihzdHJfZGV0ZWN0KGtleSwgIlF1ZXN0aW9uIikpICU+JSANCiAgICAgIGdyb3VwX2J5KEZpbGUsIFN1YmplY3RJRCwgdmFsdWUpICU+JSANCiAgICAgIHN1bW1hcmlzZShjbnQgPSBuKCksIHF1ZXN0aW9uX251bWJlciA9IHBhc3RlKGtleSwgY29sbGFwc2UgPSAiLCIpKSAlPiUgDQogICAgICBmaWx0ZXIoY250ID4gMSkgJT4lIHVuZ3JvdXAoKSwNCiAgICAgICAgICAgIGJ5ID0gInZhbHVlIg0KICAgICkgIyBUYWtpbmcgb3V0IGdlbmVyaWMgcXVlc3Rpb25zIChoYXZpbmcgc2FtZSBxdWVzdGlvbiB0ZXh0IGJ1dCBkaWZmZXJlbnQgYW5zd2VycykNCg0KDQp4LnF1ZXN0aW9ucy5kaXN0IDwtIHgucXVlc3Rpb25zICU+JSBkaXN0aW5jdCh2YWx1ZSkgJT4lIGRyb3BfbmEoKSAlPiUgIA0KICAjbXV0YXRlKFFfVU5JUVVFX0lEID0gcm93X251bWJlcigpKSAlPiUgDQogIG11dGF0ZSh0ZW1wID0gc3RyX3RyaW0oc3RyX3JlcGxhY2VfYWxsKHZhbHVlLCAiX3xcXC4iLCAiIikpKSAlPiUgDQogIA0KICBpbm5lcl9qb2luKA0KICAgIFFfcHJlLCBieSA9ICJ0ZW1wIg0KICAgIA0KICAgIA0KICApDQoNCg0KDQp4LnF1ZXN0aW9ucy5kaXN0ICU+JSB3cml0ZV9jc3YoIi4uL2RhdGEvRmlyc3RZZWFyUHJvamVjdC9RX2Rpc3RpbmN0X2lkLmNzdiIpDQp4LnF1ZXN0aW9ucy5kaXN0DQpgYGANCg0KDQpgYGB7cn0NClEgPC0geC5xdWVzdGlvbnMuZGlzdCAlPiUgZGlzdGluY3QoUV9VTklRVUVfSUQsIGxvX2lkKSAlPiUgYXJyYW5nZShRX1VOSVFVRV9JRCkgJT4lDQogIG11dGF0ZShwcmVzZW50ID0gMSkgJT4lDQogIA0KICBzcHJlYWQoa2V5ID0gImxvX2lkIiwgdmFsdWUgPSAicHJlc2VudCIpDQoNClEgJT4lIA0KICBtdXRhdGVfYWxsKGZ1bmN0aW9uKHgpIGlmZWxzZShpcy5uYSh4KSwgMCwgeCkpICU+JSAgDQogIHdyaXRlX2NzdigiLi4vZGF0YS9GaXJzdFllYXJQcm9qZWN0L1EuY3N2IikNClENCg0KYGBgDQoNCmBgYHtyfQ0KDQp4LmFuc3dlcnMgPC0gDQogIA0KICB4LmdhdGhlciAlPiUgZmlsdGVyKCFzdHJfZGV0ZWN0KGtleSwgIlF1ZXN0aW9uIikpDQoNCnguYW5zd2Vycw0KYGBgDQoNCiNUb3RhbCBRdWVzdGlvbnMgcHJlc2VudGVkIHRvIHN0dWRlbnRzDQo1MyBRdWVzdGlvbnMgYXJlIHJhbmRvbWx5IHByZXNlbnRlZCB0byBzdHVkZW50cw0KYGBge3J9DQp4LnF1ZXN0aW9ucyAlPiUgZGlzdGluY3Qoa2V5KQ0KYGBgDQoNCg0KYGBge3J9DQoNCngucXVlc3Rpb25zLmlkIDwtIHgucXVlc3Rpb25zICU+JSBpbm5lcl9qb2luKHgucXVlc3Rpb25zLmRpc3QpICMlPiUgbXV0YXRlKFFfVU5JUVVFX0lEICA9IGZhY3RvcihRX1VOSVFVRV9JRCkpIA0KDQp4LnF1ZXN0aW9ucy5pZA0KYGBgDQoNCiMgRmlsdGVyIG91dCBHZW5lcmljIFF1ZXN0aW9ucyANClF1ZXN0aW9ucyB3aXRoIHNhbWUgdGV4dCBidXQgZGlmZmVyZW50IEFuc3dlcnMNCmBgYHtyfQ0KDQp4LnF1ZXN0aW9ucy5pZC5maWx0ZXJkIDwtIHgucXVlc3Rpb25zLmlkICU+JSANCiAgYW50aV9qb2luKA0KICAgIHgucXVlc3Rpb25zLmlkICU+JSANCiAgICAgIGdyb3VwX2J5KEZpbGUsIFN1YmplY3RJRCwgUXVlc3Rpb24pICU+JSANCiAgICAgIHN1bW1hcmlzZShjbnQgPSBuKCksIHF1ZXN0aW9uX251bWJlciA9IHBhc3RlKGtleSwgY29sbGFwc2UgPSAiLCIpKSAlPiUgDQogICAgICBmaWx0ZXIoY250ID4gMSkgJT4lIHVuZ3JvdXAoKSwNCiAgICAgICAgICAgIGJ5ID0gIlF1ZXN0aW9uIg0KICAgICkgJT4lIHNlbGVjdCgtbG9faWQsIC1gTGVhcm5pbmcgT2JqZWN0aXZlYCkNCg0KDQp4LnF1ZXN0aW9ucy5pZC5maWx0ZXJkDQoNCmBgYA0KDQoNCldlIGhhdmUgdGhlIGNvcnJlY3QgUXVlc3Rpb25zLiBOb3cgd2UgbmVlZCB0byBhZGQgbWFya3Mgb2YgYW5zd2VycyBhZ2FpbnN0IHRoZSBxdWVzdGlvbnMuDQpgYGB7cn0NCg0KWC5wcmUgPC0geC5xdWVzdGlvbnMuaWQgJT4lIG11dGF0ZShpZCA9IHN0cl9zcGxpdChrZXksICIgIiwgc2ltcGxpZnkgPSBUUlVFKVssMl0pICU+JSANCiAgDQogIGlubmVyX2pvaW4oDQogICAgDQogICAgeC5hbnN3ZXJzICU+JSBtdXRhdGUoaWQgPSBzdHJfc3BsaXQoa2V5LCAiICIsIHNpbXBsaWZ5ID0gVFJVRSlbLDNdKSwgYnkgPSBjKCJGaWxlIiwgIlN1YmplY3RJRCIsICJpZCIpDQogICAgDQogICAgKSAlPiUgDQogIG11dGF0ZSh2YWx1ZS55ID0gYXMuaW50ZWdlcih2YWx1ZS55KSkgIyU+JSANCiAgI211dGF0ZShRX1VOSVFVRV9JRCA9IGFzLmludGVnZXIoUV9VTklRVUVfSUQpKQ0KDQojd3JpdGVfY3N2KFgucHJlLCAiWF9QcmUuY3N2IikNClgucHJlDQpgYGANCg0KYGBge3J9DQp1bmlxdWUoWC5wcmUkUV9VTklRVUVfSUQpDQpgYGANCg0KDQpgYGB7cn0NCg0KDQpYPC0gWC5wcmUgJT4lIHNlbGVjdCgta2V5LngsIC1rZXkueSwgLXZhbHVlLngsIC1pZCwgLXRlbXAsIC1sb19pZCwgLWBMZWFybmluZyBPYmplY3RpdmVgLCAtUXVlc3Rpb24gKSAlPiUgDQogIHNwcmVhZChrZXkgPSAiUV9VTklRVUVfSUQiLCB2YWx1ZSA9ICJ2YWx1ZS55IikgIA0KICANCg0Kd3JpdGVfY3N2KFgsICIuLi9kYXRhL0ZpcnN0WWVhclByb2plY3QvWC5jc3YiKQ0KWA0KYGBgDQoNCkxldCdzIHJ1biBzb21lIHRlc3QgdG8gdmVyaWZ5IFgNCmBgYHtyfQ0KDQpYICU+JSBzZWxlY3QoLUZpbGUsIC1TdWJqZWN0SUQpICU+JSBzdW1tYXJpc2VfYWxsKHN1bSwgbmEucm0gPSBUUlVFKQ0KDQpgYGANCg0KYGBge3J9DQoNClggJT4lIGdhdGhlcihrZXkgPSAiUXVlc3Rpb25JRCIsIHZhbHVlID0gIlNjb3JlIiwgLUZpbGUsIC1TdWJqZWN0SUQpDQoNCmBgYA0KDQojIEZpbHRlciBxdWVzdGlvbnMgYXNrZWQgaW4gRXhhbSBJDQoNCmBgYHtyfQ0KDQpsaWJyYXJ5KGphbml0b3IpDQpYICU+JSBmaWx0ZXIoRmlsZSA9PSAiRXhhbTFUcmlhbDEiKSAlPiUgcmVtb3ZlX2VtcHR5KC4sd2hpY2ggPSAiY29scyIpDQoNCmBgYA0KDQoNCg0KDQojIFF1ZXN0aW9ucyB3aXRoIGdvb2QgYXR0ZW1wdCBjb3VudA0KYGBge3J9DQoNCg0KcXVlc3Rpb25fYXR0ZW1wdGVkIDwtIFggJT4lIHJlbW92ZV9lbXB0eSguLHdoaWNoID0gImNvbHMiKSAlPiUgDQogIGdhdGhlcihrZXkgPSAiUXVlc3Rpb25JRCIsIHZhbHVlID0gIlNjb3JlcyIsIC1GaWxlLCAtU3ViamVjdElEKSAlPiUgDQogIGdyb3VwX2J5KEZpbGUsIFF1ZXN0aW9uSUQpICU+JQ0KICBzdW1tYXJpc2UodG90YWxfbmEgPSBzdW0oaXMubmEoU2NvcmVzKSksIHRvdGFsID0gbigpLCB0b3RhbF9hdHRlbXB0ZWQgPSB0b3RhbCAtIHRvdGFsX25hKQ0KDQpxdWVzdGlvbl9hdHRlbXB0ZWQgPC0gcXVlc3Rpb25fYXR0ZW1wdGVkICU+JSBmaWx0ZXIodG90YWxfYXR0ZW1wdGVkID49IDgpDQoNCnF1ZXN0aW9uX2F0dGVtcHRlZA0KDQojJT4lIGZpbHRlcihRdWVzdGlvbklEID09ICIxMDMiKQ0KDQpgYGANCg0KRmlsdGVyaW5nIG91dCBxdWVzdGlvbnMgd2l0aCBsZXNzZXIgYXR0ZW1wdHMNCg0KYGBge3J9DQoNClhfZmlsdGVyZWQgPC0gWCAlPiUgcmVtb3ZlX2VtcHR5KC4sd2hpY2ggPSAiY29scyIpICU+JSANCiAgZ2F0aGVyKGtleSA9ICJRdWVzdGlvbklEIiwgdmFsdWUgPSAiU2NvcmVzIiwgLUZpbGUsIC1TdWJqZWN0SUQpICU+JSBzZW1pX2pvaW4ocXVlc3Rpb25fYXR0ZW1wdGVkLCBieSA9IGMoIkZpbGUiLCAiUXVlc3Rpb25JRCIpKSAlPiUgDQogIHNwcmVhZChrZXkgPSAiUXVlc3Rpb25JRCIsIHZhbHVlID0gIlNjb3JlcyIpDQoNClhfZmlsdGVyZWQNCmBgYA0KDQoNCg0KIyBUYWtlIGF3YXkgcXVlc3Rpb25zIGFuc3dlcmVkIGxlc3MgdGhhdCA1IHRpbWVzIHBlciBleGFtDQpgYGB7cn0NClggJT4lIHJlbW92ZV9lbXB0eSguLHdoaWNoID0gImNvbHMiKSAlPiUgd3JpdGVfY3N2KCIuLi9kYXRhL0ZpcnN0WWVhclByb2plY3QvWC5jc3YiKQ0KDQpYX2ZpbHRlcmVkICU+JSByZW1vdmVfZW1wdHkoLix3aGljaCA9ICJjb2xzIikgJT4lIHdyaXRlX2NzdigiLi4vZGF0YS9GaXJzdFllYXJQcm9qZWN0L1hfZmlsdGVyZWQuY3N2IikNCmBgYA0KDQpXcml0ZSBDU1ZzIHNlcGVyYXRlIGZvciBlYWNoIHRyaWFsIHRvIGF2b2lkIGhhdmluZyBjb2x1bW5zIGZvciB0aG9zZSBxdWVzdGlvbnMgdGhhdCB3ZXJlIG5vdCBhc2tlZCBpbiBhIHRyaWFsLiBUaGlzIHdpbGwgaGVscCB0byBzaG93IHRoZSB0cnVlIHBpY3R1cmUgb2Ygc3BhcnNpdHkuIA0KDQpgYGB7ciBwYWdlZC5wcmludD1GQUxTRX0NCg0KZm4uY2xlYW4gPC0gZnVuY3Rpb24gKGRmKSB7DQogIHJldHVybihkZiAlPiUgcmVtb3ZlX2VtcHR5KC4sd2hpY2ggPSAiY29scyIpKQ0KICANCn0NCg0KDQpYLmluZGl2aWR1YWwubGlzdCA8LSBYICU+JSANCm5lc3QoLUZpbGUsIC5rZXkgPSAiWF9mdWxsIikgJT4lIA0KICBtdXRhdGUoWCA9IG1hcChYX2Z1bGwsIGZuLmNsZWFuKSwgDQogICAgICAgICBRX2Z1bGwgPSBtYXAoWF9mdWxsLCBmdW5jdGlvbihkZikgcmV0dXJuIChRKSkpDQoNClguaW5kaXZpZHVhbC5saXN0DQoNCg0KDQpgYGANCg0KDQoNCmBgYHtyfQ0KDQpYICU+JSBmaWx0ZXIoRmlsZSA9PSAiRXhhbTFUcmlhbDEiKSAlPiUgcmVtb3ZlX2VtcHR5KC4sd2hpY2ggPSAiY29scyIpDQoNCmBgYA0KDQojIE1lcmdlIHdpdGggUQ0KDQpgYGB7cn0NCg0KUQ0KDQpgYGANCg0KYGBge3IgIHBhZ2VkLnByaW50PUZBTFNFfQ0KDQpmbi5za2lsbHMgPC0gZnVuY3Rpb24gKGRmKSB7DQogIA0KICBkZiA8LSBkZiAlPiUgcmVtb3ZlX2VtcHR5KC4sd2hpY2ggPSAiY29scyIpICU+JQ0KICBnYXRoZXIoa2V5ID0gIlFfVU5JUVVFX0lEIiwgdmFsdWUgPSAiU2NvcmUiLCAtU3ViamVjdElEKSAlPiUNCiAgbXV0YXRlKFFfVU5JUVVFX0lEID0gYXMuaW50ZWdlcihRX1VOSVFVRV9JRCkpICU+JSBkaXN0aW5jdChRX1VOSVFVRV9JRCkgJT4lDQogIA0KICBpbm5lcl9qb2luKA0KICAgIFENCiAgICANCiAgKSAlPiUgcmVtb3ZlX2VtcHR5KC4sd2hpY2ggPSAiY29scyIpICAlPiUgbXV0YXRlX2FsbChmdW5jdGlvbih4KSBpZmVsc2UoaXMubmEoeCksIDAsIHgpKQ0KICANCiAgcmV0dXJuKGRmKQ0KICANCn0NCg0KDQpYLlEgPC0gWC5pbmRpdmlkdWFsLmxpc3QgJT4lIA0KICBtdXRhdGUoUSA9IG1hcChYLCBmbi5za2lsbHMpKQ0KDQoNClguUSANCg0KWCAlPiUgZmlsdGVyKEZpbGUgPT0gIkV4YW0yVHJpYWwyIikgJT4lIHJlbW92ZV9lbXB0eSguLHdoaWNoID0gImNvbHMiKSAlPiUNCiAgZ2F0aGVyKGtleSA9ICJRX1VOSVFVRV9JRCIsIHZhbHVlID0gIlNjb3JlIiwgLUZpbGUsIC1TdWJqZWN0SUQpICU+JQ0KICBtdXRhdGUoUV9VTklRVUVfSUQgPSBhcy5pbnRlZ2VyKFFfVU5JUVVFX0lEKSkgJT4lIGRpc3RpbmN0KFFfVU5JUVVFX0lEKSAlPiUNCg0KICBpbm5lcl9qb2luKA0KICAgIFEsIGJ5ID0gIlFfVU5JUVVFX0lEIg0KICAgIA0KICApICU+JSByZW1vdmVfZW1wdHkoLix3aGljaCA9ICJjb2xzIikgJT4lIG11dGF0ZV9hbGwoZnVuY3Rpb24oeCkgaWZlbHNlKGlzLm5hKHgpLCAwLCB4KSkgJT4lIHN1bW1hcmlzZV9hbGwoc3VtKQ0KDQoNCmBgYA0KDQpgYGB7cn0NClggJT4lIGZpbHRlcihGaWxlID09ICJFeGFtMVRyaWFsMSIpDQoNCmBgYA0KDQoNCmBgYHtyIH0NCg0KZm4ud3JpdGUgPC0gZnVuY3Rpb24oRmlsZSwgWF9mdWxsLCBYLCBRX2Z1bGwsIFEpIHsNCg0KICANCiAgcHJpbnQoWCkNCiAgWCAlPiUgd3JpdGVfY3N2KHBhc3RlMCgiLi4vZGF0YS9GaXJzdFllYXJQcm9qZWN0LyIsRmlsZSwiX1guY3N2IikpDQogIFEgJT4lIHdyaXRlX2NzdihwYXN0ZTAoIi4uL2RhdGEvRmlyc3RZZWFyUHJvamVjdC8iLEZpbGUsIl9RLmNzdiIpKQ0KICANCn0NCg0KI3dhbGsyKFguUSRGaWxlLCBYLlEkZGF0YV9jbGVhbiwgWC5RJGRhdGFfUV9za2lsbHMsIGZuLndyaXRlKQ0KDQpwd2FsayhYLlEsIGZuLndyaXRlKQ0KDQoNCmBgYA0KDQoNCg==